{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas version 1.0.3\n",
      "Numpy version  1.18.4\n",
      "XGBOOST version  1.1.1\n",
      "Sklearn version  0.23.1\n"
     ]
    }
   ],
   "source": [
    "import pandas \n",
    "print('Pandas version',pandas.__version__)\n",
    "import numpy\n",
    "print('Numpy version ',numpy.__version__)\n",
    "import xgboost\n",
    "print('XGBOOST version ',xgboost.__version__)\n",
    "import sklearn\n",
    "print('Sklearn version ',sklearn.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.metrics import *\n",
    "from sklearn.model_selection import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9568, 5) (38272, 5)\n"
     ]
    }
   ],
   "source": [
    "train_path       = r'D:\\av_hackathons\\machine_hack_13\\Train.csv'\n",
    "test_path        = r'D:\\av_hackathons\\machine_hack_13\\Test.csv'\n",
    "submis_path      = r'D:\\av_hackathons\\machine_hack_13\\Sample_Submission.csv'\n",
    "\n",
    "train_data       = pd.read_csv(train_path)\n",
    "test_data        = pd.read_csv(test_path)\n",
    "train_data['new_col1'] = train_data['AP']/train_data['V']\n",
    "test_data['new_col1']  = test_data['AP']/test_data['V']\n",
    "\n",
    "train_y               = train_data['PE'].values\n",
    "\n",
    "train_data            = train_data.drop(['PE'],axis=1)\n",
    "train                 = train_data.values\n",
    "test                  = test_data.values\n",
    "print(train.shape,test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation RMSE model 1 fold- 1 :  2.1135392518618112\n",
      "validation RMSE model 2 fold- 1 :  2.248138635259805\n",
      "validation RMSE model 3 fold- 1 :  1.995633933842503\n",
      "validation RMSE model 4 fold- 1 :  2.150421509140051\n",
      "validation RMSE fold- 1 :  1.9411797728068534\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 2 :  4.316387393266874\n",
      "validation RMSE model 2 fold- 2 :  4.433863668282092\n",
      "validation RMSE model 3 fold- 2 :  4.7339920182402295\n",
      "validation RMSE model 4 fold- 2 :  4.422532021557629\n",
      "validation RMSE fold- 2 :  4.338947129128778\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 3 :  2.166778548000724\n",
      "validation RMSE model 2 fold- 3 :  2.2186414884574557\n",
      "validation RMSE model 3 fold- 3 :  2.688481280704619\n",
      "validation RMSE model 4 fold- 3 :  2.3914741068328924\n",
      "validation RMSE fold- 3 :  2.1222038005570307\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 4 :  1.9983600505354355\n",
      "validation RMSE model 2 fold- 4 :  2.1015784535099136\n",
      "validation RMSE model 3 fold- 4 :  2.062255446122924\n",
      "validation RMSE model 4 fold- 4 :  1.9597519723551424\n",
      "validation RMSE fold- 4 :  1.9179594486564884\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 5 :  2.217088646574427\n",
      "validation RMSE model 2 fold- 5 :  2.114324323711097\n",
      "validation RMSE model 3 fold- 5 :  2.215822860695654\n",
      "validation RMSE model 4 fold- 5 :  2.2098187049364046\n",
      "validation RMSE fold- 5 :  2.072723504319452\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 6 :  2.10517162597635\n",
      "validation RMSE model 2 fold- 6 :  2.033319562642047\n",
      "validation RMSE model 3 fold- 6 :  2.079947861015818\n",
      "validation RMSE model 4 fold- 6 :  2.2798613895075652\n",
      "validation RMSE fold- 6 :  1.9851674468514102\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 7 :  2.2496195743680274\n",
      "validation RMSE model 2 fold- 7 :  2.087770788137931\n",
      "validation RMSE model 3 fold- 7 :  1.9463121805020482\n",
      "validation RMSE model 4 fold- 7 :  2.2650336805688713\n",
      "validation RMSE fold- 7 :  1.9319776610632857\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 8 :  2.379949673245292\n",
      "validation RMSE model 2 fold- 8 :  2.4422628920358385\n",
      "validation RMSE model 3 fold- 8 :  2.3676154889727017\n",
      "validation RMSE model 4 fold- 8 :  2.4778410095223222\n",
      "validation RMSE fold- 8 :  2.319366818968201\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 9 :  2.4366648605855157\n",
      "validation RMSE model 2 fold- 9 :  2.5283200366233434\n",
      "validation RMSE model 3 fold- 9 :  2.3802632218689825\n",
      "validation RMSE model 4 fold- 9 :  2.712018872078459\n",
      "validation RMSE fold- 9 :  2.3396678867035594\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 10 :  2.3649752443901955\n",
      "validation RMSE model 2 fold- 10 :  2.099182384746603\n",
      "validation RMSE model 3 fold- 10 :  2.192731662031745\n",
      "validation RMSE model 4 fold- 10 :  2.4725438994139397\n",
      "validation RMSE fold- 10 :  2.0774996752537023\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 11 :  3.721273717650743\n",
      "validation RMSE model 2 fold- 11 :  3.5976206881822606\n",
      "validation RMSE model 3 fold- 11 :  3.1306381301220907\n",
      "validation RMSE model 4 fold- 11 :  3.423433691733781\n",
      "validation RMSE fold- 11 :  3.1473610569642005\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 12 :  2.304184282074832\n",
      "validation RMSE model 2 fold- 12 :  2.3599234210842037\n",
      "validation RMSE model 3 fold- 12 :  2.441158464710602\n",
      "validation RMSE model 4 fold- 12 :  2.380754295401571\n",
      "validation RMSE fold- 12 :  2.252421464565251\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 13 :  2.180251865324818\n",
      "validation RMSE model 2 fold- 13 :  2.2458994035384983\n",
      "validation RMSE model 3 fold- 13 :  1.882612343246621\n",
      "validation RMSE model 4 fold- 13 :  1.9153539963607025\n",
      "validation RMSE fold- 13 :  1.8406048377101767\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 14 :  2.29229544732213\n",
      "validation RMSE model 2 fold- 14 :  2.11998032022104\n",
      "validation RMSE model 3 fold- 14 :  2.288084080439821\n",
      "validation RMSE model 4 fold- 14 :  1.9256784706980938\n",
      "validation RMSE fold- 14 :  1.9190613947214217\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 15 :  3.2742166340686087\n",
      "validation RMSE model 2 fold- 15 :  3.1092542990423446\n",
      "validation RMSE model 3 fold- 15 :  3.1548851687614587\n",
      "validation RMSE model 4 fold- 15 :  3.247301195393564\n",
      "validation RMSE fold- 15 :  3.091932145680924\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 16 :  1.90524272185904\n",
      "validation RMSE model 2 fold- 16 :  2.021122026656638\n",
      "validation RMSE model 3 fold- 16 :  2.0275231523851946\n",
      "validation RMSE model 4 fold- 16 :  2.346978339050595\n",
      "validation RMSE fold- 16 :  1.8500793127533228\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 17 :  1.7613761169208983\n",
      "validation RMSE model 2 fold- 17 :  1.8005130828310074\n",
      "validation RMSE model 3 fold- 17 :  1.7751305859461795\n",
      "validation RMSE model 4 fold- 17 :  1.815376214383999\n",
      "validation RMSE fold- 17 :  1.656252842686075\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 18 :  1.894394720029062\n",
      "validation RMSE model 2 fold- 18 :  1.991638492619583\n",
      "validation RMSE model 3 fold- 18 :  2.0390744666246334\n",
      "validation RMSE model 4 fold- 18 :  1.9314940508705805\n",
      "validation RMSE fold- 18 :  1.7835300841245674\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 19 :  2.2789693639704494\n",
      "validation RMSE model 2 fold- 19 :  2.106979068591125\n",
      "validation RMSE model 3 fold- 19 :  2.234324700319695\n",
      "validation RMSE model 4 fold- 19 :  2.1004882683724437\n",
      "validation RMSE fold- 19 :  2.066822315677711\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 20 :  1.9396179788978072\n",
      "validation RMSE model 2 fold- 20 :  1.9387717594395877\n",
      "validation RMSE model 3 fold- 20 :  2.177012669570003\n",
      "validation RMSE model 4 fold- 20 :  1.9637145999909813\n",
      "validation RMSE fold- 20 :  1.8445211157647385\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 21 :  2.6307223134623525\n",
      "validation RMSE model 2 fold- 21 :  2.667892739824884\n",
      "validation RMSE model 3 fold- 21 :  2.5785112476903205\n",
      "validation RMSE model 4 fold- 21 :  2.5709000448816943\n",
      "validation RMSE fold- 21 :  2.5179606296592314\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 22 :  3.056924792258396\n",
      "validation RMSE model 2 fold- 22 :  2.7596358030158545\n",
      "validation RMSE model 3 fold- 22 :  2.6390088407985712\n",
      "validation RMSE model 4 fold- 22 :  3.1544144826393077\n",
      "validation RMSE fold- 22 :  2.622216483066851\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 23 :  2.2093747087735993\n",
      "validation RMSE model 2 fold- 23 :  2.2154580981903202\n",
      "validation RMSE model 3 fold- 23 :  2.0813506769745973\n",
      "validation RMSE model 4 fold- 23 :  2.0479038320712704\n",
      "validation RMSE fold- 23 :  1.9853854297259859\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 24 :  2.101794540615456\n",
      "validation RMSE model 2 fold- 24 :  2.051278946876703\n",
      "validation RMSE model 3 fold- 24 :  2.0097285145396633\n",
      "validation RMSE model 4 fold- 24 :  1.9764328202402894\n",
      "validation RMSE fold- 24 :  1.9081920737738995\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 25 :  2.7412658468516256\n",
      "validation RMSE model 2 fold- 25 :  2.4592135177930783\n",
      "validation RMSE model 3 fold- 25 :  2.621279496105437\n",
      "validation RMSE model 4 fold- 25 :  2.5288697740448405\n",
      "validation RMSE fold- 25 :  2.4432954381270338\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 26 :  2.536571938640264\n",
      "validation RMSE model 2 fold- 26 :  2.444804293654208\n",
      "validation RMSE model 3 fold- 26 :  2.2648341758694963\n",
      "validation RMSE model 4 fold- 26 :  2.3336698592498575\n",
      "validation RMSE fold- 26 :  2.2489873103869726\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 27 :  2.2328488912536795\n",
      "validation RMSE model 2 fold- 27 :  2.492455738357765\n",
      "validation RMSE model 3 fold- 27 :  2.613994720035665\n",
      "validation RMSE model 4 fold- 27 :  2.640763432105688\n",
      "validation RMSE fold- 27 :  2.2405853611584927\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 28 :  2.2933244651669047\n",
      "validation RMSE model 2 fold- 28 :  2.265181325650762\n",
      "validation RMSE model 3 fold- 28 :  2.1504071652919348\n",
      "validation RMSE model 4 fold- 28 :  2.220261967905439\n",
      "validation RMSE fold- 28 :  2.0761022799785294\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 29 :  2.011331326822846\n",
      "validation RMSE model 2 fold- 29 :  1.9660269806591324\n",
      "validation RMSE model 3 fold- 29 :  2.18641260018842\n",
      "validation RMSE model 4 fold- 29 :  2.0469775653934987\n",
      "validation RMSE fold- 29 :  1.8897540897486942\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 30 :  2.6123762538329762\n",
      "validation RMSE model 2 fold- 30 :  2.5120584769464207\n",
      "validation RMSE model 3 fold- 30 :  2.739218471793031\n",
      "validation RMSE model 4 fold- 30 :  2.432242915214144\n",
      "validation RMSE fold- 30 :  2.416625578300937\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 31 :  2.024194015025935\n",
      "validation RMSE model 2 fold- 31 :  1.9549852346512628\n",
      "validation RMSE model 3 fold- 31 :  1.948781000613131\n",
      "validation RMSE model 4 fold- 31 :  1.8220242507944358\n",
      "validation RMSE fold- 31 :  1.7914529770985943\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation RMSE model 1 fold- 32 :  2.194368532745924\n",
      "validation RMSE model 2 fold- 32 :  1.7931635556915038\n",
      "validation RMSE model 3 fold- 32 :  1.924468558526987\n",
      "validation RMSE model 4 fold- 32 :  2.038871336887847\n",
      "validation RMSE fold- 32 :  1.773755025067972\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 33 :  1.987286563634511\n",
      "validation RMSE model 2 fold- 33 :  2.202648650601492\n",
      "validation RMSE model 3 fold- 33 :  2.2412711653047332\n",
      "validation RMSE model 4 fold- 33 :  2.2163195214789977\n",
      "validation RMSE fold- 33 :  1.9714919613251256\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 34 :  2.326176900075102\n",
      "validation RMSE model 2 fold- 34 :  2.3891491408366594\n",
      "validation RMSE model 3 fold- 34 :  2.2913156762931126\n",
      "validation RMSE model 4 fold- 34 :  2.380573511974313\n",
      "validation RMSE fold- 34 :  2.249422146078471\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 35 :  2.4355180381376793\n",
      "validation RMSE model 2 fold- 35 :  2.418177232224573\n",
      "validation RMSE model 3 fold- 35 :  2.365308080269771\n",
      "validation RMSE model 4 fold- 35 :  2.4691357592175116\n",
      "validation RMSE fold- 35 :  2.32658693775635\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 36 :  2.456886345174656\n",
      "validation RMSE model 2 fold- 36 :  2.425874971091947\n",
      "validation RMSE model 3 fold- 36 :  2.155820345856343\n",
      "validation RMSE model 4 fold- 36 :  2.268908420170833\n",
      "validation RMSE fold- 36 :  2.1465642157233216\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 37 :  2.0510128528794747\n",
      "validation RMSE model 2 fold- 37 :  2.0588039848836024\n",
      "validation RMSE model 3 fold- 37 :  1.9980922443476787\n",
      "validation RMSE model 4 fold- 37 :  2.0970127324288863\n",
      "validation RMSE fold- 37 :  1.9154612483164848\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 38 :  1.977408772116866\n",
      "validation RMSE model 2 fold- 38 :  1.9409571214417687\n",
      "validation RMSE model 3 fold- 38 :  1.9414577470794732\n",
      "validation RMSE model 4 fold- 38 :  1.820673748129422\n",
      "validation RMSE fold- 38 :  1.7879738909954321\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 39 :  2.005081094280576\n",
      "validation RMSE model 2 fold- 39 :  2.0708859632005625\n",
      "validation RMSE model 3 fold- 39 :  2.1908793320112694\n",
      "validation RMSE model 4 fold- 39 :  2.076242898386167\n",
      "validation RMSE fold- 39 :  1.9641163613593329\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 40 :  2.785860799885161\n",
      "validation RMSE model 2 fold- 40 :  2.8267671823048026\n",
      "validation RMSE model 3 fold- 40 :  2.9471482888108076\n",
      "validation RMSE model 4 fold- 40 :  2.8947110569327856\n",
      "validation RMSE fold- 40 :  2.749870230215166\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 41 :  2.297386958225032\n",
      "validation RMSE model 2 fold- 41 :  2.2345065443385286\n",
      "validation RMSE model 3 fold- 41 :  2.3790394523041036\n",
      "validation RMSE model 4 fold- 41 :  2.1909021982524437\n",
      "validation RMSE fold- 41 :  2.1428842940549666\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 42 :  2.3745360462659417\n",
      "validation RMSE model 2 fold- 42 :  2.310180344526112\n",
      "validation RMSE model 3 fold- 42 :  2.4118018834347703\n",
      "validation RMSE model 4 fold- 42 :  2.3915334729112327\n",
      "validation RMSE fold- 42 :  2.267076861162938\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 43 :  1.94153120097456\n",
      "validation RMSE model 2 fold- 43 :  1.9076506931461905\n",
      "validation RMSE model 3 fold- 43 :  1.8690786514877624\n",
      "validation RMSE model 4 fold- 43 :  1.7568460366072922\n",
      "validation RMSE fold- 43 :  1.7407134421124022\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 44 :  2.156683133296991\n",
      "validation RMSE model 2 fold- 44 :  2.105344210200822\n",
      "validation RMSE model 3 fold- 44 :  2.1870784785106956\n",
      "validation RMSE model 4 fold- 44 :  2.0689283727216563\n",
      "validation RMSE fold- 44 :  2.023426516111672\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 45 :  2.1028105883038686\n",
      "validation RMSE model 2 fold- 45 :  2.1148648240841714\n",
      "validation RMSE model 3 fold- 45 :  2.133956048141373\n",
      "validation RMSE model 4 fold- 45 :  2.0990872161370557\n",
      "validation RMSE fold- 45 :  2.004032659468245\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 46 :  2.3457943973533317\n",
      "validation RMSE model 2 fold- 46 :  2.7916577358564725\n",
      "validation RMSE model 3 fold- 46 :  2.2540641441384066\n",
      "validation RMSE model 4 fold- 46 :  2.3365223909920183\n",
      "validation RMSE fold- 46 :  2.2152202916002017\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 47 :  2.1689564874717697\n",
      "validation RMSE model 2 fold- 47 :  2.212174205038765\n",
      "validation RMSE model 3 fold- 47 :  2.380397732128297\n",
      "validation RMSE model 4 fold- 47 :  2.285031196643028\n",
      "validation RMSE fold- 47 :  2.127872470774479\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 48 :  2.043671929988036\n",
      "validation RMSE model 2 fold- 48 :  2.3595299484972565\n",
      "validation RMSE model 3 fold- 48 :  1.900050138905894\n",
      "validation RMSE model 4 fold- 48 :  2.2753300596269397\n",
      "validation RMSE fold- 48 :  1.869456616120477\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 49 :  2.332358118310388\n",
      "validation RMSE model 2 fold- 49 :  2.3240127530164076\n",
      "validation RMSE model 3 fold- 49 :  2.2962905723400597\n",
      "validation RMSE model 4 fold- 49 :  2.2605411403465094\n",
      "validation RMSE fold- 49 :  2.1962502323081603\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 50 :  1.859600673332109\n",
      "validation RMSE model 2 fold- 50 :  1.9115324051626994\n",
      "validation RMSE model 3 fold- 50 :  1.9074263325060707\n",
      "validation RMSE model 4 fold- 50 :  1.9647447077414981\n",
      "validation RMSE fold- 50 :  1.7938924448400393\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 51 :  2.683176384854305\n",
      "validation RMSE model 2 fold- 51 :  2.752723597226248\n",
      "validation RMSE model 3 fold- 51 :  2.681662914405194\n",
      "validation RMSE model 4 fold- 51 :  2.7548303455307677\n",
      "validation RMSE fold- 51 :  2.6414429982657612\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 52 :  2.6106057810209395\n",
      "validation RMSE model 2 fold- 52 :  2.2180579322493927\n",
      "validation RMSE model 3 fold- 52 :  2.349457444284238\n",
      "validation RMSE model 4 fold- 52 :  2.3920350702978763\n",
      "validation RMSE fold- 52 :  2.1978490150974315\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 53 :  2.005268638168833\n",
      "validation RMSE model 2 fold- 53 :  1.7261705617580572\n",
      "validation RMSE model 3 fold- 53 :  1.7829500260856632\n",
      "validation RMSE model 4 fold- 53 :  1.8038498232512445\n",
      "validation RMSE fold- 53 :  1.676799201538331\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 54 :  2.401590831989533\n",
      "validation RMSE model 2 fold- 54 :  2.7392858629501835\n",
      "validation RMSE model 3 fold- 54 :  2.407112622238201\n",
      "validation RMSE model 4 fold- 54 :  2.3566248537519066\n",
      "validation RMSE fold- 54 :  2.2965723735009513\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 55 :  2.0397970655287834\n",
      "validation RMSE model 2 fold- 55 :  2.1548586236589364\n",
      "validation RMSE model 3 fold- 55 :  2.106377605989745\n",
      "validation RMSE model 4 fold- 55 :  2.0577666172747393\n",
      "validation RMSE fold- 55 :  1.9707316474975196\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 56 :  2.176341424094766\n",
      "validation RMSE model 2 fold- 56 :  2.010935107433702\n",
      "validation RMSE model 3 fold- 56 :  1.8956985061884162\n",
      "validation RMSE model 4 fold- 56 :  2.0926253765533596\n",
      "validation RMSE fold- 56 :  1.864248162251789\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 57 :  2.1637639596448506\n",
      "validation RMSE model 2 fold- 57 :  2.220574794117002\n",
      "validation RMSE model 3 fold- 57 :  2.340449203301215\n",
      "validation RMSE model 4 fold- 57 :  2.4015863881195294\n",
      "validation RMSE fold- 57 :  2.1248194427700002\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 58 :  1.7835102984103217\n",
      "validation RMSE model 2 fold- 58 :  1.6912825802385287\n",
      "validation RMSE model 3 fold- 58 :  1.8751287297421395\n",
      "validation RMSE model 4 fold- 58 :  1.8622102825648634\n",
      "validation RMSE fold- 58 :  1.681277795940673\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 59 :  2.676666595182037\n",
      "validation RMSE model 2 fold- 59 :  2.571119434866593\n",
      "validation RMSE model 3 fold- 59 :  2.502195784090043\n",
      "validation RMSE model 4 fold- 59 :  2.679802694846168\n",
      "validation RMSE fold- 59 :  2.4941816391662055\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 60 :  2.91603164402484\n",
      "validation RMSE model 2 fold- 60 :  2.9533354549224162\n",
      "validation RMSE model 3 fold- 60 :  2.9701210646466154\n",
      "validation RMSE model 4 fold- 60 :  2.9000464983528107\n",
      "validation RMSE fold- 60 :  2.858293433933626\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 61 :  2.8164250952295546\n",
      "validation RMSE model 2 fold- 61 :  2.4048604905641295\n",
      "validation RMSE model 3 fold- 61 :  2.7915554801919487\n",
      "validation RMSE model 4 fold- 61 :  2.315963020608679\n",
      "validation RMSE fold- 61 :  2.2786119545334818\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 62 :  2.1049312919091427\n",
      "validation RMSE model 2 fold- 62 :  2.282022256800884\n",
      "validation RMSE model 3 fold- 62 :  2.248303421650763\n",
      "validation RMSE model 4 fold- 62 :  2.189425438129864\n",
      "validation RMSE fold- 62 :  2.094639654696908\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation RMSE model 1 fold- 63 :  2.2546467864868815\n",
      "validation RMSE model 2 fold- 63 :  2.1498551444085363\n",
      "validation RMSE model 3 fold- 63 :  2.490186652708036\n",
      "validation RMSE model 4 fold- 63 :  2.24569220550041\n",
      "validation RMSE fold- 63 :  2.1219491268202364\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 64 :  2.0736342848991707\n",
      "validation RMSE model 2 fold- 64 :  2.1838273804093835\n",
      "validation RMSE model 3 fold- 64 :  2.296103484241993\n",
      "validation RMSE model 4 fold- 64 :  2.074013313133603\n",
      "validation RMSE fold- 64 :  2.0016683333836713\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 65 :  2.5164220472065733\n",
      "validation RMSE model 2 fold- 65 :  2.6206446071803255\n",
      "validation RMSE model 3 fold- 65 :  2.3916432216126173\n",
      "validation RMSE model 4 fold- 65 :  2.682736822847257\n",
      "validation RMSE fold- 65 :  2.381298990767681\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 66 :  2.0195196723592153\n",
      "validation RMSE model 2 fold- 66 :  2.0919421541593226\n",
      "validation RMSE model 3 fold- 66 :  1.9804000160970647\n",
      "validation RMSE model 4 fold- 66 :  2.0301547930198076\n",
      "validation RMSE fold- 66 :  1.9336715745291098\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 67 :  2.771201112416397\n",
      "validation RMSE model 2 fold- 67 :  2.716311622956433\n",
      "validation RMSE model 3 fold- 67 :  2.913668343742429\n",
      "validation RMSE model 4 fold- 67 :  2.653117229014627\n",
      "validation RMSE fold- 67 :  2.6386923980634442\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 68 :  2.2114085463893787\n",
      "validation RMSE model 2 fold- 68 :  2.1343041939915475\n",
      "validation RMSE model 3 fold- 68 :  2.1510127694609977\n",
      "validation RMSE model 4 fold- 68 :  2.2118380706760163\n",
      "validation RMSE fold- 68 :  2.081339221188713\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 69 :  2.0881750847745897\n",
      "validation RMSE model 2 fold- 69 :  2.171898738286206\n",
      "validation RMSE model 3 fold- 69 :  2.06634996719083\n",
      "validation RMSE model 4 fold- 69 :  2.2084372193616333\n",
      "validation RMSE fold- 69 :  2.0220925756577643\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 70 :  2.4526167089618953\n",
      "validation RMSE model 2 fold- 70 :  2.5490559165311315\n",
      "validation RMSE model 3 fold- 70 :  2.5936874364626887\n",
      "validation RMSE model 4 fold- 70 :  2.6918486108758\n",
      "validation RMSE fold- 70 :  2.362624638587254\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 71 :  2.54609541921604\n",
      "validation RMSE model 2 fold- 71 :  2.5405524432018267\n",
      "validation RMSE model 3 fold- 71 :  2.5334087311163964\n",
      "validation RMSE model 4 fold- 71 :  2.501434766808641\n",
      "validation RMSE fold- 71 :  2.4102699944773116\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 72 :  2.84736074035904\n",
      "validation RMSE model 2 fold- 72 :  3.0782493828791626\n",
      "validation RMSE model 3 fold- 72 :  2.756098046600706\n",
      "validation RMSE model 4 fold- 72 :  2.7493822710501425\n",
      "validation RMSE fold- 72 :  2.7118605914129583\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 73 :  2.682570498697617\n",
      "validation RMSE model 2 fold- 73 :  2.5052249665423534\n",
      "validation RMSE model 3 fold- 73 :  2.6709602535750014\n",
      "validation RMSE model 4 fold- 73 :  2.445281939075756\n",
      "validation RMSE fold- 73 :  2.425877182920269\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 74 :  3.426932552487962\n",
      "validation RMSE model 2 fold- 74 :  3.1501703138389257\n",
      "validation RMSE model 3 fold- 74 :  3.3515939791546177\n",
      "validation RMSE model 4 fold- 74 :  3.3024909351877976\n",
      "validation RMSE fold- 74 :  3.189136117620843\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 75 :  2.230208824844438\n",
      "validation RMSE model 2 fold- 75 :  2.233743645263905\n",
      "validation RMSE model 3 fold- 75 :  2.1131048338824985\n",
      "validation RMSE model 4 fold- 75 :  2.1046910048880867\n",
      "validation RMSE fold- 75 :  2.0201071265454504\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 76 :  2.8965171719700145\n",
      "validation RMSE model 2 fold- 76 :  3.086659304585455\n",
      "validation RMSE model 3 fold- 76 :  3.093490028376834\n",
      "validation RMSE model 4 fold- 76 :  2.7238326294475455\n",
      "validation RMSE fold- 76 :  2.740180086740289\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 77 :  3.2105426326434485\n",
      "validation RMSE model 2 fold- 77 :  3.3596448325140527\n",
      "validation RMSE model 3 fold- 77 :  3.3308326803631885\n",
      "validation RMSE model 4 fold- 77 :  3.400205199819022\n",
      "validation RMSE fold- 77 :  3.2122151393172453\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 78 :  2.2649676316840046\n",
      "validation RMSE model 2 fold- 78 :  2.0965347703336277\n",
      "validation RMSE model 3 fold- 78 :  2.3881123042994425\n",
      "validation RMSE model 4 fold- 78 :  2.355469213362258\n",
      "validation RMSE fold- 78 :  2.0891945357541166\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 79 :  1.7546676425252254\n",
      "validation RMSE model 2 fold- 79 :  1.9294552464969428\n",
      "validation RMSE model 3 fold- 79 :  1.7792174550554067\n",
      "validation RMSE model 4 fold- 79 :  1.761817114200999\n",
      "validation RMSE fold- 79 :  1.6757083917302176\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 80 :  2.6893902558382528\n",
      "validation RMSE model 2 fold- 80 :  2.65716749264161\n",
      "validation RMSE model 3 fold- 80 :  2.737030188060449\n",
      "validation RMSE model 4 fold- 80 :  2.8967343128438325\n",
      "validation RMSE fold- 80 :  2.6100243282913027\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 81 :  2.498554258768611\n",
      "validation RMSE model 2 fold- 81 :  2.593673712425297\n",
      "validation RMSE model 3 fold- 81 :  2.7431967945566536\n",
      "validation RMSE model 4 fold- 81 :  2.521088867739356\n",
      "validation RMSE fold- 81 :  2.4750419684199465\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 82 :  2.2220984322599575\n",
      "validation RMSE model 2 fold- 82 :  2.3929038144883843\n",
      "validation RMSE model 3 fold- 82 :  2.24433268698902\n",
      "validation RMSE model 4 fold- 82 :  2.328442766335191\n",
      "validation RMSE fold- 82 :  2.175615086019159\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 83 :  2.2223771255536406\n",
      "validation RMSE model 2 fold- 83 :  1.955379566929933\n",
      "validation RMSE model 3 fold- 83 :  2.086123564835536\n",
      "validation RMSE model 4 fold- 83 :  2.2770029632311832\n",
      "validation RMSE fold- 83 :  1.9267929119437301\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 84 :  2.2386428684852726\n",
      "validation RMSE model 2 fold- 84 :  2.45601963057605\n",
      "validation RMSE model 3 fold- 84 :  2.19653416327529\n",
      "validation RMSE model 4 fold- 84 :  2.389077747952638\n",
      "validation RMSE fold- 84 :  2.1671118085027037\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 85 :  2.3653005369625095\n",
      "validation RMSE model 2 fold- 85 :  2.1924095870921945\n",
      "validation RMSE model 3 fold- 85 :  2.3965749926173934\n",
      "validation RMSE model 4 fold- 85 :  2.3743365847522417\n",
      "validation RMSE fold- 85 :  2.209110812127573\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 86 :  2.7135438282650135\n",
      "validation RMSE model 2 fold- 86 :  2.742697948978556\n",
      "validation RMSE model 3 fold- 86 :  2.6509924526040707\n",
      "validation RMSE model 4 fold- 86 :  2.6698228653844747\n",
      "validation RMSE fold- 86 :  2.607236848126885\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 87 :  1.9751037442722805\n",
      "validation RMSE model 2 fold- 87 :  2.100869425256423\n",
      "validation RMSE model 3 fold- 87 :  2.1356784134121756\n",
      "validation RMSE model 4 fold- 87 :  2.056002205107516\n",
      "validation RMSE fold- 87 :  1.9087169886799966\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 88 :  1.9842421087551212\n",
      "validation RMSE model 2 fold- 88 :  1.74353680289475\n",
      "validation RMSE model 3 fold- 88 :  1.8869231496800616\n",
      "validation RMSE model 4 fold- 88 :  2.0287612615130644\n",
      "validation RMSE fold- 88 :  1.7249774625411018\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 89 :  2.245810963439473\n",
      "validation RMSE model 2 fold- 89 :  2.237870865738711\n",
      "validation RMSE model 3 fold- 89 :  2.6693156573898587\n",
      "validation RMSE model 4 fold- 89 :  2.1070017392661287\n",
      "validation RMSE fold- 89 :  2.0870436747006713\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 90 :  2.6148045618290836\n",
      "validation RMSE model 2 fold- 90 :  2.733431287357012\n",
      "validation RMSE model 3 fold- 90 :  2.6650939181166673\n",
      "validation RMSE model 4 fold- 90 :  2.5396409639259043\n",
      "validation RMSE fold- 90 :  2.508650273446251\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 91 :  5.00746234149153\n",
      "validation RMSE model 2 fold- 91 :  5.029509001799242\n",
      "validation RMSE model 3 fold- 91 :  5.127139832908685\n",
      "validation RMSE model 4 fold- 91 :  4.979493309278442\n",
      "validation RMSE fold- 91 :  4.976298279198706\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 92 :  2.1187136311606762\n",
      "validation RMSE model 2 fold- 92 :  2.0037871660790865\n",
      "validation RMSE model 3 fold- 92 :  1.9931754778291921\n",
      "validation RMSE model 4 fold- 92 :  2.0695668116340387\n",
      "validation RMSE fold- 92 :  1.9248928408926005\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 93 :  2.4647829546896998\n",
      "validation RMSE model 2 fold- 93 :  2.3532895141337544\n",
      "validation RMSE model 3 fold- 93 :  2.532522957617095\n",
      "validation RMSE model 4 fold- 93 :  2.6236592102628244\n",
      "validation RMSE fold- 93 :  2.326630483860639\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation RMSE model 1 fold- 94 :  2.259291792488894\n",
      "validation RMSE model 2 fold- 94 :  2.3117472539186354\n",
      "validation RMSE model 3 fold- 94 :  2.38193019194235\n",
      "validation RMSE model 4 fold- 94 :  2.2773049917610306\n",
      "validation RMSE fold- 94 :  2.161623390681395\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 95 :  2.557592667805985\n",
      "validation RMSE model 2 fold- 95 :  2.696455412457316\n",
      "validation RMSE model 3 fold- 95 :  2.450753352054897\n",
      "validation RMSE model 4 fold- 95 :  2.6123014464599423\n",
      "validation RMSE fold- 95 :  2.426661950332957\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 96 :  2.58827792254564\n",
      "validation RMSE model 2 fold- 96 :  2.1929984671429685\n",
      "validation RMSE model 3 fold- 96 :  2.2502909558064115\n",
      "validation RMSE model 4 fold- 96 :  2.267402575598449\n",
      "validation RMSE fold- 96 :  2.1403676316889357\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 97 :  2.0460903240558195\n",
      "validation RMSE model 2 fold- 97 :  1.9546953363075263\n",
      "validation RMSE model 3 fold- 97 :  2.1930786025725486\n",
      "validation RMSE model 4 fold- 97 :  2.1982884202887423\n",
      "validation RMSE fold- 97 :  1.8957453949113243\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 98 :  1.9427729670059966\n",
      "validation RMSE model 2 fold- 98 :  2.032372413875396\n",
      "validation RMSE model 3 fold- 98 :  2.2184299188163865\n",
      "validation RMSE model 4 fold- 98 :  2.2300089288064524\n",
      "validation RMSE fold- 98 :  1.9210652488027935\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 99 :  2.4086142827616275\n",
      "validation RMSE model 2 fold- 99 :  2.070036670114684\n",
      "validation RMSE model 3 fold- 99 :  2.1324079006010015\n",
      "validation RMSE model 4 fold- 99 :  2.194300862477118\n",
      "validation RMSE fold- 99 :  2.040628381500348\n",
      "\n",
      "\n",
      "validation RMSE model 1 fold- 100 :  2.169123804141227\n",
      "validation RMSE model 2 fold- 100 :  1.9727916530426715\n",
      "validation RMSE model 3 fold- 100 :  2.031186123795517\n",
      "validation RMSE model 4 fold- 100 :  2.2090522644024526\n",
      "validation RMSE fold- 100 :  1.9112291808918782\n",
      "\n",
      "\n",
      "OOF RMSE:-  2.274297211088118\n"
     ]
    }
   ],
   "source": [
    "oof_pred               = np.zeros((len(train), ))\n",
    "y_pred_1               = np.zeros((len(test), ))\n",
    "y_pred_2               = np.zeros((len(test), ))\n",
    "y_pred_3               = np.zeros((len(test), ))\n",
    "y_pred_4               = np.zeros((len(test), ))\n",
    "num_models             = 4\n",
    "\n",
    "n_splits               = 100\n",
    "error                  = []\n",
    "kf                     = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "\n",
    "for fold, (tr_ind, val_ind) in enumerate(kf.split(train, train_y)):\n",
    "    wghts = [0]*num_models\n",
    "    rmse  = []\n",
    "    X_train, X_val     = train[tr_ind], train[val_ind]\n",
    "    y_train, y_val     = train_y[tr_ind], train_y[val_ind]\n",
    "    \n",
    "    model1 = XGBRegressor(n_estimators=400,random_state=42,subsample=0.8)\n",
    "    model1.fit(X_train,np.log(y_train))\n",
    "    val_pred1 = model1.predict(X_val)\n",
    "    rmse.append(np.sqrt(mean_squared_error(y_val, np.exp(val_pred1))))\n",
    "    y_pred_1  += model1.predict(test) / (n_splits)\n",
    "    \n",
    "    model2 = XGBRegressor(n_estimators=500,random_state=42,subsample=0.9)\n",
    "    model2.fit(X_train,np.log(y_train))\n",
    "    val_pred2 = model2.predict(X_val)\n",
    "    rmse.append(np.sqrt(mean_squared_error(y_val, np.exp(val_pred2))))\n",
    "    y_pred_2  += model2.predict(test) / (n_splits)\n",
    "    \n",
    "    model3 = XGBRegressor(n_estimators=600,random_state=42,subsample=0.7)\n",
    "    model3.fit(X_train,np.log(y_train))\n",
    "    val_pred3 = model3.predict(X_val)\n",
    "    rmse.append(np.sqrt(mean_squared_error(y_val, np.exp(val_pred3))))\n",
    "    y_pred_3  += model3.predict(test) / (n_splits)\n",
    "    \n",
    "    model4     = XGBRegressor(n_estimators=700,random_state=42,subsample=0.7,reg_lambda=2)\n",
    "    model4.fit(X_train,np.log(y_train))\n",
    "    val_pred4  = model4.predict(X_val)\n",
    "    rmse.append(np.sqrt(mean_squared_error(y_val, np.exp(val_pred4))))\n",
    "    y_pred_4  += model4.predict(test) / (n_splits)\n",
    "    \n",
    "    \n",
    "    print('validation RMSE model 1 fold-',fold+1,': ',rmse[0])\n",
    "    print('validation RMSE model 2 fold-',fold+1,': ',rmse[1])\n",
    "    print('validation RMSE model 3 fold-',fold+1,': ',rmse[2])\n",
    "    print('validation RMSE model 4 fold-',fold+1,': ',rmse[3])\n",
    "    \n",
    "    wghts = np.exp(-100*np.array(rmse/sum(rmse)))\n",
    "    wghts = wghts/sum(wghts)\n",
    "    \n",
    "    val_pred           = wghts[0]*val_pred1+wghts[1]*val_pred2+wghts[2]*val_pred3+wghts[3]*val_pred4\n",
    "\n",
    "    print('validation RMSE fold-',fold+1,': ',np.sqrt(mean_squared_error(y_val, np.exp(val_pred))))\n",
    "    error.append(np.sqrt(mean_squared_error(y_val, np.exp(val_pred))))\n",
    "    oof_pred[val_ind]  = np.exp(val_pred)\n",
    "    print('\\n')\n",
    "    \n",
    "y_pred_final = np.exp(wghts[0]*y_pred_1+\n",
    "                      wghts[1]*y_pred_2+\n",
    "                      wghts[2]*y_pred_3+\n",
    "                      wghts[3]*y_pred_4\n",
    "                     )\n",
    "print('OOF RMSE:- ',np.sqrt(mean_squared_error(oof_pred,train_y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4859194450382616"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.std(error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df       = pd.read_csv(r'D:\\av_hackathons\\machine_hack_13\\sample_submission.csv')\n",
    "submission_df['PE'] = y_pred_final\n",
    "submission_df.to_csv('xgb_blending_within_folds.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
